<!DOCTYPE html><html><head>
      <title>RNN</title>
      <meta charset="utf-8">
      <meta name="viewport" content="width=device-width, initial-scale=1.0">
      
      <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.11.1/dist/katex.min.css">
      
      
      
      <script type="text/javascript" src="https://cdn.jsdelivr.net/npm/mermaid@8.5.2/dist/mermaid.min.js"></script>
      
      
      
      
      
      <style>
      /**
 * prism.js Github theme based on GitHub's theme.
 * @author Sam Clarke
 */
code[class*="language-"],
pre[class*="language-"] {
  color: #333;
  background: none;
  font-family: Consolas, "Liberation Mono", Menlo, Courier, monospace;
  text-align: left;
  white-space: pre;
  word-spacing: normal;
  word-break: normal;
  word-wrap: normal;
  line-height: 1.4;

  -moz-tab-size: 8;
  -o-tab-size: 8;
  tab-size: 8;

  -webkit-hyphens: none;
  -moz-hyphens: none;
  -ms-hyphens: none;
  hyphens: none;
}

/* Code blocks */
pre[class*="language-"] {
  padding: .8em;
  overflow: auto;
  /* border: 1px solid #ddd; */
  border-radius: 3px;
  /* background: #fff; */
  background: #f5f5f5;
}

/* Inline code */
:not(pre) > code[class*="language-"] {
  padding: .1em;
  border-radius: .3em;
  white-space: normal;
  background: #f5f5f5;
}

.token.comment,
.token.blockquote {
  color: #969896;
}

.token.cdata {
  color: #183691;
}

.token.doctype,
.token.punctuation,
.token.variable,
.token.macro.property {
  color: #333;
}

.token.operator,
.token.important,
.token.keyword,
.token.rule,
.token.builtin {
  color: #a71d5d;
}

.token.string,
.token.url,
.token.regex,
.token.attr-value {
  color: #183691;
}

.token.property,
.token.number,
.token.boolean,
.token.entity,
.token.atrule,
.token.constant,
.token.symbol,
.token.command,
.token.code {
  color: #0086b3;
}

.token.tag,
.token.selector,
.token.prolog {
  color: #63a35c;
}

.token.function,
.token.namespace,
.token.pseudo-element,
.token.class,
.token.class-name,
.token.pseudo-class,
.token.id,
.token.url-reference .token.variable,
.token.attr-name {
  color: #795da3;
}

.token.entity {
  cursor: help;
}

.token.title,
.token.title .token.punctuation {
  font-weight: bold;
  color: #1d3e81;
}

.token.list {
  color: #ed6a43;
}

.token.inserted {
  background-color: #eaffea;
  color: #55a532;
}

.token.deleted {
  background-color: #ffecec;
  color: #bd2c00;
}

.token.bold {
  font-weight: bold;
}

.token.italic {
  font-style: italic;
}


/* JSON */
.language-json .token.property {
  color: #183691;
}

.language-markup .token.tag .token.punctuation {
  color: #333;
}

/* CSS */
code.language-css,
.language-css .token.function {
  color: #0086b3;
}

/* YAML */
.language-yaml .token.atrule {
  color: #63a35c;
}

code.language-yaml {
  color: #183691;
}

/* Ruby */
.language-ruby .token.function {
  color: #333;
}

/* Markdown */
.language-markdown .token.url {
  color: #795da3;
}

/* Makefile */
.language-makefile .token.symbol {
  color: #795da3;
}

.language-makefile .token.variable {
  color: #183691;
}

.language-makefile .token.builtin {
  color: #0086b3;
}

/* Bash */
.language-bash .token.keyword {
  color: #0086b3;
}

/* highlight */
pre[data-line] {
  position: relative;
  padding: 1em 0 1em 3em;
}
pre[data-line] .line-highlight-wrapper {
  position: absolute;
  top: 0;
  left: 0;
  background-color: transparent;
  display: block;
  width: 100%;
}

pre[data-line] .line-highlight {
  position: absolute;
  left: 0;
  right: 0;
  padding: inherit 0;
  margin-top: 1em;
  background: hsla(24, 20%, 50%,.08);
  background: linear-gradient(to right, hsla(24, 20%, 50%,.1) 70%, hsla(24, 20%, 50%,0));
  pointer-events: none;
  line-height: inherit;
  white-space: pre;
}

pre[data-line] .line-highlight:before, 
pre[data-line] .line-highlight[data-end]:after {
  content: attr(data-start);
  position: absolute;
  top: .4em;
  left: .6em;
  min-width: 1em;
  padding: 0 .5em;
  background-color: hsla(24, 20%, 50%,.4);
  color: hsl(24, 20%, 95%);
  font: bold 65%/1.5 sans-serif;
  text-align: center;
  vertical-align: .3em;
  border-radius: 999px;
  text-shadow: none;
  box-shadow: 0 1px white;
}

pre[data-line] .line-highlight[data-end]:after {
  content: attr(data-end);
  top: auto;
  bottom: .4em;
}html body{font-family:"Helvetica Neue",Helvetica,"Segoe UI",Arial,freesans,sans-serif;font-size:16px;line-height:1.6;color:#333;background-color:#fff;overflow:initial;box-sizing:border-box;word-wrap:break-word}html body>:first-child{margin-top:0}html body h1,html body h2,html body h3,html body h4,html body h5,html body h6{line-height:1.2;margin-top:1em;margin-bottom:16px;color:#000}html body h1{font-size:2.25em;font-weight:300;padding-bottom:.3em}html body h2{font-size:1.75em;font-weight:400;padding-bottom:.3em}html body h3{font-size:1.5em;font-weight:500}html body h4{font-size:1.25em;font-weight:600}html body h5{font-size:1.1em;font-weight:600}html body h6{font-size:1em;font-weight:600}html body h1,html body h2,html body h3,html body h4,html body h5{font-weight:600}html body h5{font-size:1em}html body h6{color:#5c5c5c}html body strong{color:#000}html body del{color:#5c5c5c}html body a:not([href]){color:inherit;text-decoration:none}html body a{color:#08c;text-decoration:none}html body a:hover{color:#00a3f5;text-decoration:none}html body img{max-width:100%}html body>p{margin-top:0;margin-bottom:16px;word-wrap:break-word}html body>ul,html body>ol{margin-bottom:16px}html body ul,html body ol{padding-left:2em}html body ul.no-list,html body ol.no-list{padding:0;list-style-type:none}html body ul ul,html body ul ol,html body ol ol,html body ol ul{margin-top:0;margin-bottom:0}html body li{margin-bottom:0}html body li.task-list-item{list-style:none}html body li>p{margin-top:0;margin-bottom:0}html body .task-list-item-checkbox{margin:0 .2em .25em -1.8em;vertical-align:middle}html body .task-list-item-checkbox:hover{cursor:pointer}html body blockquote{margin:16px 0;font-size:inherit;padding:0 15px;color:#5c5c5c;background-color:#f0f0f0;border-left:4px solid #d6d6d6}html body blockquote>:first-child{margin-top:0}html body blockquote>:last-child{margin-bottom:0}html body hr{height:4px;margin:32px 0;background-color:#d6d6d6;border:0 none}html body table{margin:10px 0 15px 0;border-collapse:collapse;border-spacing:0;display:block;width:100%;overflow:auto;word-break:normal;word-break:keep-all}html body table th{font-weight:bold;color:#000}html body table td,html body table th{border:1px solid #d6d6d6;padding:6px 13px}html body dl{padding:0}html body dl dt{padding:0;margin-top:16px;font-size:1em;font-style:italic;font-weight:bold}html body dl dd{padding:0 16px;margin-bottom:16px}html body code{font-family:Menlo,Monaco,Consolas,'Courier New',monospace;font-size:.85em !important;color:#000;background-color:#f0f0f0;border-radius:3px;padding:.2em 0}html body code::before,html body code::after{letter-spacing:-0.2em;content:"\00a0"}html body pre>code{padding:0;margin:0;font-size:.85em !important;word-break:normal;white-space:pre;background:transparent;border:0}html body .highlight{margin-bottom:16px}html body .highlight pre,html body pre{padding:1em;overflow:auto;font-size:.85em !important;line-height:1.45;border:#d6d6d6;border-radius:3px}html body .highlight pre{margin-bottom:0;word-break:normal}html body pre code,html body pre tt{display:inline;max-width:initial;padding:0;margin:0;overflow:initial;line-height:inherit;word-wrap:normal;background-color:transparent;border:0}html body pre code:before,html body pre tt:before,html body pre code:after,html body pre tt:after{content:normal}html body p,html body blockquote,html body ul,html body ol,html body dl,html body pre{margin-top:0;margin-bottom:16px}html body kbd{color:#000;border:1px solid #d6d6d6;border-bottom:2px solid #c7c7c7;padding:2px 4px;background-color:#f0f0f0;border-radius:3px}@media print{html body{background-color:#fff}html body h1,html body h2,html body h3,html body h4,html body h5,html body h6{color:#000;page-break-after:avoid}html body blockquote{color:#5c5c5c}html body pre{page-break-inside:avoid}html body table{display:table}html body img{display:block;max-width:100%;max-height:100%}html body pre,html body code{word-wrap:break-word;white-space:pre}}.markdown-preview{width:100%;height:100%;box-sizing:border-box}.markdown-preview .pagebreak,.markdown-preview .newpage{page-break-before:always}.markdown-preview pre.line-numbers{position:relative;padding-left:3.8em;counter-reset:linenumber}.markdown-preview pre.line-numbers>code{position:relative}.markdown-preview pre.line-numbers .line-numbers-rows{position:absolute;pointer-events:none;top:1em;font-size:100%;left:0;width:3em;letter-spacing:-1px;border-right:1px solid #999;-webkit-user-select:none;-moz-user-select:none;-ms-user-select:none;user-select:none}.markdown-preview pre.line-numbers .line-numbers-rows>span{pointer-events:none;display:block;counter-increment:linenumber}.markdown-preview pre.line-numbers .line-numbers-rows>span:before{content:counter(linenumber);color:#999;display:block;padding-right:.8em;text-align:right}.markdown-preview .mathjax-exps .MathJax_Display{text-align:center !important}.markdown-preview:not([for="preview"]) .code-chunk .btn-group{display:none}.markdown-preview:not([for="preview"]) .code-chunk .status{display:none}.markdown-preview:not([for="preview"]) .code-chunk .output-div{margin-bottom:16px}.scrollbar-style::-webkit-scrollbar{width:8px}.scrollbar-style::-webkit-scrollbar-track{border-radius:10px;background-color:transparent}.scrollbar-style::-webkit-scrollbar-thumb{border-radius:5px;background-color:rgba(150,150,150,0.66);border:4px solid rgba(150,150,150,0.66);background-clip:content-box}html body[for="html-export"]:not([data-presentation-mode]){position:relative;width:100%;height:100%;top:0;left:0;margin:0;padding:0;overflow:auto}html body[for="html-export"]:not([data-presentation-mode]) .markdown-preview{position:relative;top:0}@media screen and (min-width:914px){html body[for="html-export"]:not([data-presentation-mode]) .markdown-preview{padding:2em calc(50% - 457px + 2em)}}@media screen and (max-width:914px){html body[for="html-export"]:not([data-presentation-mode]) .markdown-preview{padding:2em}}@media screen and (max-width:450px){html body[for="html-export"]:not([data-presentation-mode]) .markdown-preview{font-size:14px !important;padding:1em}}@media print{html body[for="html-export"]:not([data-presentation-mode]) #sidebar-toc-btn{display:none}}html body[for="html-export"]:not([data-presentation-mode]) #sidebar-toc-btn{position:fixed;bottom:8px;left:8px;font-size:28px;cursor:pointer;color:inherit;z-index:99;width:32px;text-align:center;opacity:.4}html body[for="html-export"]:not([data-presentation-mode])[html-show-sidebar-toc] #sidebar-toc-btn{opacity:1}html body[for="html-export"]:not([data-presentation-mode])[html-show-sidebar-toc] .md-sidebar-toc{position:fixed;top:0;left:0;width:300px;height:100%;padding:32px 0 48px 0;font-size:14px;box-shadow:0 0 4px rgba(150,150,150,0.33);box-sizing:border-box;overflow:auto;background-color:inherit}html body[for="html-export"]:not([data-presentation-mode])[html-show-sidebar-toc] .md-sidebar-toc::-webkit-scrollbar{width:8px}html body[for="html-export"]:not([data-presentation-mode])[html-show-sidebar-toc] .md-sidebar-toc::-webkit-scrollbar-track{border-radius:10px;background-color:transparent}html body[for="html-export"]:not([data-presentation-mode])[html-show-sidebar-toc] .md-sidebar-toc::-webkit-scrollbar-thumb{border-radius:5px;background-color:rgba(150,150,150,0.66);border:4px solid rgba(150,150,150,0.66);background-clip:content-box}html body[for="html-export"]:not([data-presentation-mode])[html-show-sidebar-toc] .md-sidebar-toc a{text-decoration:none}html body[for="html-export"]:not([data-presentation-mode])[html-show-sidebar-toc] .md-sidebar-toc ul{padding:0 1.6em;margin-top:.8em}html body[for="html-export"]:not([data-presentation-mode])[html-show-sidebar-toc] .md-sidebar-toc li{margin-bottom:.8em}html body[for="html-export"]:not([data-presentation-mode])[html-show-sidebar-toc] .md-sidebar-toc ul{list-style-type:none}html body[for="html-export"]:not([data-presentation-mode])[html-show-sidebar-toc] .markdown-preview{left:300px;width:calc(100% -  300px);padding:2em calc(50% - 457px -  150px);margin:0;box-sizing:border-box}@media screen and (max-width:1274px){html body[for="html-export"]:not([data-presentation-mode])[html-show-sidebar-toc] .markdown-preview{padding:2em}}@media screen and (max-width:450px){html body[for="html-export"]:not([data-presentation-mode])[html-show-sidebar-toc] .markdown-preview{width:100%}}html body[for="html-export"]:not([data-presentation-mode]):not([html-show-sidebar-toc]) .markdown-preview{left:50%;transform:translateX(-50%)}html body[for="html-export"]:not([data-presentation-mode]):not([html-show-sidebar-toc]) .md-sidebar-toc{display:none}
/* Please visit the URL below for more information: */
/*   https://shd101wyy.github.io/markdown-preview-enhanced/#/customize-css */

      </style>
    </head>
    <body for="html-export">
      <div class="mume markdown-preview  ">
      <link rel="stylesheet" href="../../style/index.css">
<script src="../../style/index.js"></script>
<h1 class="mume-header" id="%E6%9C%BA%E5%99%A8%E5%AD%A6%E4%B9%A0rnnindexhtml"><a href="./index.html">&#x673A;&#x5668;&#x5B66;&#x4E60;&#xFF1A;RNN</a></h1>

<ul>
<li><a href="#%E6%9C%BA%E5%99%A8%E5%AD%A6%E4%B9%A0rnnindexhtml">&#x673A;&#x5668;&#x5B66;&#x4E60;&#xFF1A;RNN</a>
<ul>
<li><a href="#%E5%88%9D%E8%AF%86%E5%BE%AA%E7%8E%AF%E7%A5%9E%E7%BB%8F%E7%BD%91%E7%BB%9C">&#x521D;&#x8BC6;&#x5FAA;&#x73AF;&#x795E;&#x7ECF;&#x7F51;&#x7EDC;</a>
<ul>
<li><a href="#%E9%9A%90%E8%97%8F%E5%B1%82%E5%B1%95%E5%BC%80">&#x9690;&#x85CF;&#x5C42;&#x5C55;&#x5F00;</a></li>
<li><a href="#%E5%BE%AA%E7%8E%AF%E4%BD%93">&#x5FAA;&#x73AF;&#x4F53;</a></li>
<li><a href="#%E7%BC%BA%E9%99%B7">&#x7F3A;&#x9677;</a></li>
</ul>
</li>
<li><a href="#lstm">LSTM</a>
<ul>
<li><a href="#%E5%8D%95%E5%85%83%E6%95%B0%E9%87%8Fnum-units">&#x5355;&#x5143;&#x6570;&#x91CF;&#xFF08;num units&#xFF09;</a></li>
<li><a href="#%E5%8D%95%E5%85%83%E7%8A%B6%E6%80%81">&#x5355;&#x5143;&#x72B6;&#x6001;</a></li>
<li><a href="#%E9%97%B8%E9%97%A8">&#x95F8;&#x95E8;</a>
<ul>
<li><a href="#%E5%BF%98%E8%AE%B0%E9%97%A8">&#x5FD8;&#x8BB0;&#x95E8;</a></li>
<li><a href="#%E8%BE%93%E5%85%A5%E9%97%A8">&#x8F93;&#x5165;&#x95E8;</a>
<ul>
<li><a href="#%E6%9B%B4%E6%96%B0%E6%97%A7%E5%8D%95%E5%85%83%E7%8A%B6%E6%80%81">&#x66F4;&#x65B0;&#x65E7;&#x5355;&#x5143;&#x72B6;&#x6001;</a></li>
</ul>
</li>
<li><a href="#%E8%BE%93%E5%87%BA%E9%97%A8">&#x8F93;&#x51FA;&#x95E8;</a></li>
</ul>
</li>
<li><a href="#lstm%E5%8F%98%E4%BD%93">LSTM&#x53D8;&#x4F53;</a></li>
</ul>
</li>
</ul>
</li>
</ul>
<h2 class="mume-header" id="%E5%88%9D%E8%AF%86%E5%BE%AA%E7%8E%AF%E7%A5%9E%E7%BB%8F%E7%BD%91%E7%BB%9C">&#x521D;&#x8BC6;&#x5FAA;&#x73AF;&#x795E;&#x7ECF;&#x7F51;&#x7EDC;</h2>

<p>&#x4EBA;&#x7C7B;&#x4E0D;&#x4F1A;&#x6BCF;&#x79D2;&#x90FD;&#x4ECE;&#x5934;&#x5F00;&#x59CB;&#x601D;&#x8003;&#xFF01;</p>
<p>&#x5FAA;&#x73AF;&#x795E;&#x7ECF;&#x7F51;&#x7EDC;&#x4F1A;&#x8BB0;&#x5FC6;&#x4E4B;&#x524D;&#x7684;&#x4FE1;&#x606F;&#xFF0C;&#x5E76;&#x5229;&#x7528;&#x4E4B;&#x524D;&#x7684;&#x4FE1;&#x606F;&#x5F71;&#x54CD;&#x540E;&#x9762;&#x7ED3;&#x70B9;&#x7684;&#x8F93;&#x51FA;&#x3002;&#x4E5F;&#x5C31;&#x662F;&#x8BF4;&#xFF0C;&#x5FAA;&#x73AF;&#x795E;&#x7ECF;&#x7F51;&#x7EDC;&#x7684;&#x9690;&#x85CF;&#x5C42;&#x4E4B;&#x95F4;&#x7684;&#x7ED3;&#x70B9;&#x662F;&#x6709;&#x8FDE;&#x63A5;&#x7684;&#x3002;</p>
<div style="text-align: center">
<p><img src="./images/rnn_rolled.png" alt="&#x9012;&#x5F52;&#x795E;&#x7ECF;&#x7F51;&#x7EDC;&#x5177;&#x6709;&#x5FAA;&#x73AF;"><br>
<span class="highlight"><em>&#x9690;&#x85CF;&#x5C42;&#x7684;&#x8F93;&#x5165;&#xFF0C;&#x4E0D;&#x4EC5;&#x5305;&#x62EC;&#x8F93;&#x5165;&#x5C42;&#x7684;&#x8F93;&#x51FA;&#xFF0C;<br>
&#x8FD8;&#x5305;&#x62EC;&#x4E0A;&#x4E00;&#x65F6;&#x523B;&#x9690;&#x85CF;&#x5C42;&#x7684;&#x8F93;&#x51FA;</em></span></p>
</div>
<p>&#x9700;&#x8981;&#x7279;&#x522B;&#x6CE8;&#x610F;&#x7684;&#x662F;&#xFF0C;RNN&#x7684;&#x9690;&#x85CF;&#x5C42;&#x5E76;&#x4E0D;&#x548C;&#x4F20;&#x7EDF;&#x7684;&#x524D;&#x9988;&#x795E;&#x7ECF;&#x7F51;&#x7EDC;&#x5177;&#x6709;&#x76F8;&#x540C;&#x7684;&#x542B;&#x4E49;&#xFF0C;&#x5728;&#x4E0A;&#x56FE;&#x4E2D;&#xFF0C;A&#x5373;&#x4E3A;RNN&#x7684;&#x9690;&#x85CF;&#x5C42;&#x3002;</p>
<h3 class="mume-header" id="%E9%9A%90%E8%97%8F%E5%B1%82%E5%B1%95%E5%BC%80">&#x9690;&#x85CF;&#x5C42;&#x5C55;&#x5F00;</h3>

<p><img src="./images/rnn_unrolled.png" alt="&#x5C55;&#x5F00;&#x7684;&#x9012;&#x5F52;&#x795E;&#x7ECF;&#x7F51;&#x7EDC;"></p>
<p>&#x5FAA;&#x73AF;&#x795E;&#x7ECF;&#x7F51;&#x7EDC;&#x7684;&#x9690;&#x85CF;&#x5C42;&#x53EF;&#x4EE5;&#x770B;&#x4F5C;&#x662F;&#x540C;&#x4E00;&#x7F51;&#x7EDC;&#x7684;&#x591A;&#x4E2A;&#x526F;&#x672C;&#xFF0C;&#x6BCF;&#x4E2A;&#x526F;&#x672C;&#x90FD;&#x5C06;&#x6D88;&#x606F;&#x4F20;&#x9012;&#x7ED9;&#x540E;&#x7EE7;&#x8005;&#x3002;&#x5728;&#x795E;&#x7ECF;&#x7F51;&#x7EDC;&#x903B;&#x8F91;&#x4E2D;&#xFF0C;&#x4E0A;&#x56FE;&#x5176;&#x5B9E;&#x53EA;&#x6709;&#x5DE6;&#x4FA7;&#x90E8;&#x5206;&#xFF0C;&#x5176;&#x5C55;&#x5F00;&#x53EF;&#x7406;&#x89E3;&#x4E3A;&#x5728;&#x4E0D;&#x540C;&#x65F6;&#x95F4;&#xFF08;<strong>time step</strong>&#xFF09;&#x4E0B;&#x540C;&#x4E00;&#x9690;&#x5C42;&#xFF08;<strong>cell</strong>&#xFF09;&#x7684;&#x72B6;&#x6001;&#xFF0C;&#x660E;&#x663E;&#x5B83;&#x4EEC;&#x662F;&#x53C2;&#x6570;&#x5171;&#x4EAB;&#x7684;&#x3002;</p>
<h3 class="mume-header" id="%E5%BE%AA%E7%8E%AF%E4%BD%93">&#x5FAA;&#x73AF;&#x4F53;</h3>

<p>&#x5FAA;&#x73AF;&#x795E;&#x7ECF;&#x7F51;&#x7EDC;&#x53EF;&#x4EE5;&#x770B;&#x4F5C;&#x662F;&#x540C;&#x4E00;&#x795E;&#x7ECF;&#x7F51;&#x7EDC;&#x7ED3;&#x6784;&#x5728;&#x65F6;&#x95F4;&#x5E8F;&#x5217;&#x4E0A;&#x88AB;&#x590D;&#x5236;&#x591A;&#x6B21;&#x7684;&#x7ED3;&#x679C;&#xFF0C;&#x8FD9;&#x4E2A;&#x88AB;&#x590D;&#x5236;&#x591A;&#x6B21;&#x7684;&#x7ED3;&#x6784;&#x88AB;&#x79F0;&#x4E4B;&#x4E3A;<strong>&#x5FAA;&#x73AF;&#x4F53;</strong>&#x3002;</p>
<p>&#x5982;&#x4F55;&#x8BBE;&#x8BA1;&#x5FAA;&#x73AF;&#x4F53;&#x7684;&#x7F51;&#x7EDC;&#x7ED3;&#x6784;&#x662F;&#x5FAA;&#x73AF;&#x795E;&#x7ECF;&#x7F51;&#x7EDC;&#x89E3;&#x51B3;&#x5B9E;&#x9645;&#x95EE;&#x9898;&#x7684;&#x5173;&#x952E;&#x3002;</p>
<div style="text-align: center">
<p><img src="./images/rnn_simple.png" alt="&#x6807;&#x51C6;RNN&#x91CD;&#x590D;&#x6A21;&#x5757;&#x7ED3;&#x6784;"><br>
<em>&#x5728;&#x6807;&#x51C6;RNN&#x4E2D;&#xFF0C;&#x6B64;&#x91CD;&#x590D;&#x6A21;&#x5757;&#x5C06;&#x5177;&#x6709;&#x975E;&#x5E38;&#x7B80;&#x5355;&#x7684;&#x7ED3;&#x6784;&#xFF08;tanh&#x5C42;&#xFF09;</em></p>
</div>
<h3 class="mume-header" id="%E7%BC%BA%E9%99%B7">&#x7F3A;&#x9677;</h3>

<p>&#x6709;&#x65F6;&#xFF0C;&#x6211;&#x4EEC;&#x53EA;&#x9700;&#x8981;&#x67E5;&#x770B;&#x6700;&#x65B0;&#x4FE1;&#x606F;&#x5373;&#x53EF;&#x6267;&#x884C;&#x5F53;&#x524D;&#x4EFB;&#x52A1;&#x3002;</p>
<p><img src="./images/rnn_shorttermdepdencies.png" alt="&#x77ED;&#x671F;&#x4F9D;&#x8D56;"></p>
<p>&#x4F46;&#x662F;&#x5728;&#x67D0;&#x4E9B;&#x60C5;&#x51B5;&#x4E0B;&#xFF0C;&#x6211;&#x4EEC;&#x9700;&#x8981;&#x66F4;&#x591A;&#x7684;&#x4E0A;&#x4E0B;&#x6587;&#x3002;&#x4E0D;&#x5E78;&#x7684;&#x662F;&#xFF0C;&#x968F;&#x7740;&#x5DEE;&#x8DDD;&#x7684;&#x6269;&#x5927;&#xFF0C;RNN&#x53D8;&#x5F97;&#x65E0;&#x6CD5;&#x5B66;&#x4E60;&#x8FDE;&#x63A5;&#x4FE1;&#x606F;&#x3002;</p>
<p><img src="./images/rnn_longtermdependencies.png" alt="&#x957F;&#x671F;&#x4F9D;&#x8D56;"></p>
<h2 class="mume-header" id="lstm">LSTM</h2>

<blockquote>
<p><a href="https://colah.github.io/posts/2015-08-Understanding-LSTMs/">Understanding-LSTMs</a></p>
</blockquote>
<blockquote>
<ul>
<li><span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><mi>&#x3C3;</mi></mrow><annotation encoding="application/x-tex">&#x3C3;</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:0.43056em;vertical-align:0em;"></span><span class="mord mathdefault" style="margin-right:0.03588em;">&#x3C3;</span></span></span></span>&#xFF1A;&#x6FC0;&#x6D3B;&#x51FD;&#x6570; <span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><mrow><mi mathvariant="normal">S</mi><mi mathvariant="normal">i</mi><mi mathvariant="normal">g</mi><mi mathvariant="normal">m</mi><mi mathvariant="normal">o</mi><mi mathvariant="normal">i</mi><mi mathvariant="normal">d</mi></mrow><mo stretchy="false">(</mo><mi>x</mi><mo stretchy="false">)</mo><mo>=</mo><mfrac><mn>1</mn><mrow><mn>1</mn><mo>+</mo><msup><mi>e</mi><mrow><mo>&#x2212;</mo><mi>x</mi></mrow></msup></mrow></mfrac></mrow><annotation encoding="application/x-tex">\mathrm{Sigmoid}(x) = \frac{1}{1 + e^{-x}}</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:1em;vertical-align:-0.25em;"></span><span class="mord"><span class="mord mathrm">S</span><span class="mord mathrm">i</span><span class="mord mathrm" style="margin-right:0.01389em;">g</span><span class="mord mathrm">m</span><span class="mord mathrm">o</span><span class="mord mathrm">i</span><span class="mord mathrm">d</span></span><span class="mopen">(</span><span class="mord mathdefault">x</span><span class="mclose">)</span><span class="mspace" style="margin-right:0.2777777777777778em;"></span><span class="mrel">=</span><span class="mspace" style="margin-right:0.2777777777777778em;"></span></span><span class="base"><span class="strut" style="height:1.2484389999999999em;vertical-align:-0.403331em;"></span><span class="mord"><span class="mopen nulldelimiter"></span><span class="mfrac"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.845108em;"><span style="top:-2.655em;"><span class="pstrut" style="height:3em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mtight"><span class="mord mtight">1</span><span class="mbin mtight">+</span><span class="mord mtight"><span class="mord mathdefault mtight">e</span><span class="msupsub"><span class="vlist-t"><span class="vlist-r"><span class="vlist" style="height:0.7026642857142857em;"><span style="top:-2.786em;margin-right:0.07142857142857144em;"><span class="pstrut" style="height:2.5em;"></span><span class="sizing reset-size3 size1 mtight"><span class="mord mtight"><span class="mord mtight">&#x2212;</span><span class="mord mathdefault mtight">x</span></span></span></span></span></span></span></span></span></span></span></span><span style="top:-3.23em;"><span class="pstrut" style="height:3em;"></span><span class="frac-line" style="border-bottom-width:0.04em;"></span></span><span style="top:-3.394em;"><span class="pstrut" style="height:3em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mtight"><span class="mord mtight">1</span></span></span></span></span><span class="vlist-s">&#x200B;</span></span><span class="vlist-r"><span class="vlist" style="height:0.403331em;"><span></span></span></span></span></span><span class="mclose nulldelimiter"></span></span></span></span></span></li>
<li><span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><mi>t</mi><mi>a</mi><mi>n</mi><mi>h</mi></mrow><annotation encoding="application/x-tex">tanh</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:0.69444em;vertical-align:0em;"></span><span class="mord mathdefault">t</span><span class="mord mathdefault">a</span><span class="mord mathdefault">n</span><span class="mord mathdefault">h</span></span></span></span>&#xFF1A;&#x6FC0;&#x6D3B;&#x51FD;&#x6570; <span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><mrow><mi mathvariant="normal">T</mi><mi mathvariant="normal">a</mi><mi mathvariant="normal">n</mi><mi mathvariant="normal">h</mi></mrow><mo stretchy="false">(</mo><mi>x</mi><mo stretchy="false">)</mo><mo>=</mo><mfrac><mrow><msup><mi>e</mi><mi>x</mi></msup><mo>&#x2212;</mo><msup><mi>e</mi><mrow><mo>&#x2212;</mo><mi>x</mi></mrow></msup></mrow><mrow><msup><mi>e</mi><mi>x</mi></msup><mo>+</mo><msup><mi>e</mi><mrow><mo>&#x2212;</mo><mi>x</mi></mrow></msup></mrow></mfrac></mrow><annotation encoding="application/x-tex">\mathrm{Tanh}(x) = \frac{e^{x} - e^{-x}}{e^{x} + e^{-x}}</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:1em;vertical-align:-0.25em;"></span><span class="mord"><span class="mord mathrm">T</span><span class="mord mathrm">a</span><span class="mord mathrm">n</span><span class="mord mathrm">h</span></span><span class="mopen">(</span><span class="mord mathdefault">x</span><span class="mclose">)</span><span class="mspace" style="margin-right:0.2777777777777778em;"></span><span class="mrel">=</span><span class="mspace" style="margin-right:0.2777777777777778em;"></span></span><span class="base"><span class="strut" style="height:1.3906960000000002em;vertical-align:-0.403331em;"></span><span class="mord"><span class="mopen nulldelimiter"></span><span class="mfrac"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.987365em;"><span style="top:-2.655em;"><span class="pstrut" style="height:3em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mtight"><span class="mord mtight"><span class="mord mathdefault mtight">e</span><span class="msupsub"><span class="vlist-t"><span class="vlist-r"><span class="vlist" style="height:0.5935428571428571em;"><span style="top:-2.786em;margin-right:0.07142857142857144em;"><span class="pstrut" style="height:2.5em;"></span><span class="sizing reset-size3 size1 mtight"><span class="mord mtight"><span class="mord mathdefault mtight">x</span></span></span></span></span></span></span></span></span><span class="mbin mtight">+</span><span class="mord mtight"><span class="mord mathdefault mtight">e</span><span class="msupsub"><span class="vlist-t"><span class="vlist-r"><span class="vlist" style="height:0.7026642857142857em;"><span style="top:-2.786em;margin-right:0.07142857142857144em;"><span class="pstrut" style="height:2.5em;"></span><span class="sizing reset-size3 size1 mtight"><span class="mord mtight"><span class="mord mtight">&#x2212;</span><span class="mord mathdefault mtight">x</span></span></span></span></span></span></span></span></span></span></span></span><span style="top:-3.23em;"><span class="pstrut" style="height:3em;"></span><span class="frac-line" style="border-bottom-width:0.04em;"></span></span><span style="top:-3.394em;"><span class="pstrut" style="height:3em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mtight"><span class="mord mtight"><span class="mord mathdefault mtight">e</span><span class="msupsub"><span class="vlist-t"><span class="vlist-r"><span class="vlist" style="height:0.7385428571428572em;"><span style="top:-2.931em;margin-right:0.07142857142857144em;"><span class="pstrut" style="height:2.5em;"></span><span class="sizing reset-size3 size1 mtight"><span class="mord mtight"><span class="mord mathdefault mtight">x</span></span></span></span></span></span></span></span></span><span class="mbin mtight">&#x2212;</span><span class="mord mtight"><span class="mord mathdefault mtight">e</span><span class="msupsub"><span class="vlist-t"><span class="vlist-r"><span class="vlist" style="height:0.8476642857142858em;"><span style="top:-2.931em;margin-right:0.07142857142857144em;"><span class="pstrut" style="height:2.5em;"></span><span class="sizing reset-size3 size1 mtight"><span class="mord mtight"><span class="mord mtight">&#x2212;</span><span class="mord mathdefault mtight">x</span></span></span></span></span></span></span></span></span></span></span></span></span><span class="vlist-s">&#x200B;</span></span><span class="vlist-r"><span class="vlist" style="height:0.403331em;"><span></span></span></span></span></span><span class="mclose nulldelimiter"></span></span></span></span></span></li>
<li><span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><mo>&#x2297;</mo></mrow><annotation encoding="application/x-tex">&#x2297;</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:0.66666em;vertical-align:-0.08333em;"></span><span class="mord">&#x2297;</span></span></span></span>&#xFF1A;&#x8F93;&#x5165;&#x90E8;&#x5206;&#x76F8;&#x4E58;</li>
<li><span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><mo>&#x2295;</mo></mrow><annotation encoding="application/x-tex">&#x2295;</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:0.66666em;vertical-align:-0.08333em;"></span><span class="mord">&#x2295;</span></span></span></span>&#xFF1A;&#x8F93;&#x5165;&#x90E8;&#x5206;&#x76F8;&#x52A0;</li>
</ul>
</blockquote>
<p>&#x957F;&#x77ED;&#x671F;&#x8BB0;&#x5FC6;&#x7F51;&#x7EDC;&#xFF08;LSTM&#xFF09;&#x662F;&#x4E00;&#x79CD;&#x7279;&#x6B8A;&#x7684;RNN&#xFF0C;&#x80FD;&#x591F;&#x5B66;&#x4E60;&#x957F;&#x671F;&#x4F9D;&#x8D56;&#x5173;&#x7CFB;&#x3002;</p>
<p><img src="./images/rnn_lstm_chain.png" alt="LSTM"></p>
<div class="mermaid">graph LR
h1_t-1[&quot;h_(t-1)&quot;]
x1_t[&quot;X_(t)&quot;]
c1_t-1[&quot;C_(t-1)&quot;]
stake11((&quot;&#xB7;&quot;))
input11[input1]
input12[input2]

h1_t-1--&gt;stake11
x1_t--&gt;stake11
c1_t-1--&gt;input12
stake11--&gt;input11

subgraph Cell
    f1_t[&quot;f_(t)&quot;]
    i1_t[&quot;i_(t)&quot;]
    cc1_t[&quot;C~_(t)&quot;]
    o1_t[&quot;o_(t)&quot;]
    c1_t[&quot;C_(t)&quot;]
    h1_t[&quot;h_(t)&quot;]

    input11--Sigmod--&gt;f1_t
    input11--Sigmod--&gt;i1_t
    input11--Tanh--&gt;cc1_t
    input11--Sigmod--&gt;o1_t

    mul11((&quot;&#xD7;&quot;))
    mul12((&quot;&#xD7;&quot;))
    mul13((&quot;&#xD7;&quot;))
    add11((&quot;+&quot;))

    input12--&gt;mul11
    f1_t--&gt;mul11
    i1_t--&gt;mul12
    cc1_t--&gt;mul12
    mul11--&gt;add11
    mul12--&gt;add11
    add11--&gt;c1_t

    c1_t--Tanh--&gt;mul13
    o1_t--&gt;mul13
    mul13--&gt;h1_t
end

h1_t--&gt;output11[&quot;output1&quot;]
c1_t--&gt;output12[&quot;output2&quot;]

</div><p>&#xA0;</p>
<div class="mermaid">graph TB
g1((&quot;&#xB7;&quot;))---note1[&quot;&#x5C5E;&#x6027;&#x5408;&#x5E76;&quot;]
g2((&quot;+&quot;))---note2[&quot;&#x77E9;&#x9635;&#x52A0;&#x6CD5;&quot;]
g3((&quot;&#xD7;&quot;))---note3[&quot;&#x77E9;&#x9635;&#x4E58;&#x6CD5;&quot;]
</div><h3 class="mume-header" id="%E5%8D%95%E5%85%83%E6%95%B0%E9%87%8Fnum-units">&#x5355;&#x5143;&#x6570;&#x91CF;&#xFF08;num units&#xFF09;</h3>

<p>&#x5373;&#x6307;<span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><mi>h</mi><mo stretchy="false">(</mo><mi>t</mi><mo stretchy="false">)</mo></mrow><annotation encoding="application/x-tex">h(t)</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:1em;vertical-align:-0.25em;"></span><span class="mord mathdefault">h</span><span class="mopen">(</span><span class="mord mathdefault">t</span><span class="mclose">)</span></span></span></span>&#x7684;&#x7EF4;&#x6570;&#x3002;</p>
<h3 class="mume-header" id="%E5%8D%95%E5%85%83%E7%8A%B6%E6%80%81">&#x5355;&#x5143;&#x72B6;&#x6001;</h3>

<p>&#x5355;&#x5143;&#x72B6;&#x6001;&#x6709;&#x70B9;&#x50CF;&#x4F20;&#x9001;&#x5E26;&#x3002;&#x5B83;&#x6CBF;&#x6574;&#x4E2A;&#x94FE;&#x6761;&#x4E00;&#x76F4;&#x6CBF;&#x76F4;&#x7EBF;&#x79FB;&#x52A8;&#xFF0C;&#x53EA;&#x6709;&#x4E00;&#x4E9B;&#x8F83;&#x5C0F;&#x7684;&#x7EBF;&#x6027;&#x76F8;&#x4E92;&#x4F5C;&#x7528;&#x3002;&#x4FE1;&#x606F;&#x975E;&#x5E38;&#x5BB9;&#x6613;&#x5C31;&#x53EF;&#x4EE5;&#x4E0D;&#x52A0;&#x6539;&#x53D8;&#x5730;&#x6D41;&#x52A8;&#x3002;</p>
<p><img src="./images/rnn_lstm_C-line.png" alt="&#x5355;&#x5143;&#x72B6;&#x6001;"></p>
<h3 class="mume-header" id="%E9%97%B8%E9%97%A8">&#x95F8;&#x95E8;</h3>

<p>LSTM&#x6709;&#x5220;&#x9664;&#x4FE1;&#x606F;&#x6216;&#x5C06;&#x4FE1;&#x606F;&#x6DFB;&#x52A0;&#x5230;&#x5355;&#x5143;&#x72B6;&#x6001;&#x7684;&#x80FD;&#x529B;&#xFF0C;&#x8FD9;&#x4E9B;&#x4FE1;&#x606F;&#x7531;&#x79F0;&#x4E3A;&#x95E8;&#x7684;&#x7ED3;&#x6784;&#x7CBE;&#x5FC3;&#x8C03;&#x8282;&#x3002;</p>
<div style="text-align: center">
<p><img src="./images/rnn_lstm_gate.png" alt="&#x95F8;&#x95E8;"><br>
<em>&#x95E8;&#x72B6;&#x7ED3;&#x6784;</em></p>
</div>
<p>&#x95F8;&#x95E8;&#x662F;&#x4E00;&#x79CD;&#x9009;&#x62E9;&#x6027;&#x5730;&#x8BA9;&#x4FE1;&#x606F;&#x901A;&#x8FC7;&#x7684;&#x65B9;&#x5F0F;&#x3002;&#x5176;&#x6709;&#x4EE5;&#x4E0B;&#x4E09;&#x79CD;&#x95E8;&#x3002;</p>
<h4 class="mume-header" id="%E5%BF%98%E8%AE%B0%E9%97%A8">&#x5FD8;&#x8BB0;&#x95E8;</h4>

<p>LSTM&#x7684;&#x7B2C;&#x4E00;&#x6B65;&#x662F;&#x51B3;&#x5B9A;&#x8981;&#x4ECE;&#x5355;&#x5143;&#x72B6;&#x6001;&#x4E2D;&#x4E22;&#x5F03;&#x54EA;&#x4E9B;&#x4FE1;&#x606F;&#x3002;</p>
<p><img src="./images/rnn_lstm_focus-f.png" alt></p>
<h4 class="mume-header" id="%E8%BE%93%E5%85%A5%E9%97%A8">&#x8F93;&#x5165;&#x95E8;</h4>

<p>&#x4E0B;&#x4E00;&#x6B65;&#x662F;&#x786E;&#x5B9A;&#x8981;&#x5728;&#x5355;&#x5143;&#x72B6;&#x6001;&#x4E0B;&#x5B58;&#x50A8;&#x54EA;&#x4E9B;&#x65B0;&#x4FE1;&#x606F;&#x3002;&#x5305;&#x542B;&#x4E24;&#x90E8;&#x5206;&#xFF0C;&#x7B2C;&#x4E00;&#x8DEF;&#xFF08;&#x3C3;&#xFF09;&#x51B3;&#x5B9A;&#x66F4;&#x65B0;&#x54EA;&#x4E9B;&#x503C;&#xFF0C;&#x7B2C;&#x4E8C;&#x8DEF;&#xFF08;tanh&#xFF09;&#x65B0;&#x5EFA;&#x5019;&#x9009;&#x503C;&#x5411;&#x91CF;&#x5E76;&#x6DFB;&#x52A0;&#x5230;&#x72B6;&#x6001;&#x4E2D;&#x3002;</p>
<p><img src="./images/rnn_lstm_focus-i.png" alt="&#x8F93;&#x5165;&#x95E8;"></p>
<h5 class="mume-header" id="%E6%9B%B4%E6%96%B0%E6%97%A7%E5%8D%95%E5%85%83%E7%8A%B6%E6%80%81">&#x66F4;&#x65B0;&#x65E7;&#x5355;&#x5143;&#x72B6;&#x6001;</h5>

<p>&#x5C06;&#x5355;&#x5143;&#x65E7;&#x72B6;&#x6001;&#x4E0E;<span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><msub><mi>f</mi><mi>t</mi></msub></mrow><annotation encoding="application/x-tex">f_t</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:0.8888799999999999em;vertical-align:-0.19444em;"></span><span class="mord"><span class="mord mathdefault" style="margin-right:0.10764em;">f</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.2805559999999999em;"><span style="top:-2.5500000000000003em;margin-left:-0.10764em;margin-right:0.05em;"><span class="pstrut" style="height:2.7em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mathdefault mtight">t</span></span></span></span><span class="vlist-s">&#x200B;</span></span><span class="vlist-r"><span class="vlist" style="height:0.15em;"><span></span></span></span></span></span></span></span></span></span>&#x76F8;&#x4E58;&#xFF0C;&#x7136;&#x540E;&#x52A0;&#x4E0A;<span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><msub><mi>i</mi><mi>t</mi></msub><mo>&#xD7;</mo><msub><mover accent="true"><mi>C</mi><mo>~</mo></mover><mi>t</mi></msub></mrow><annotation encoding="application/x-tex">i_t &#xD7; \tilde{C}_t</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:0.80952em;vertical-align:-0.15em;"></span><span class="mord"><span class="mord mathdefault">i</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.2805559999999999em;"><span style="top:-2.5500000000000003em;margin-left:0em;margin-right:0.05em;"><span class="pstrut" style="height:2.7em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mathdefault mtight">t</span></span></span></span><span class="vlist-s">&#x200B;</span></span><span class="vlist-r"><span class="vlist" style="height:0.15em;"><span></span></span></span></span></span></span><span class="mspace" style="margin-right:0.2222222222222222em;"></span><span class="mbin">&#xD7;</span><span class="mspace" style="margin-right:0.2222222222222222em;"></span></span><span class="base"><span class="strut" style="height:1.0701899999999998em;vertical-align:-0.15em;"></span><span class="mord"><span class="mord accent"><span class="vlist-t"><span class="vlist-r"><span class="vlist" style="height:0.9201899999999998em;"><span style="top:-3em;"><span class="pstrut" style="height:3em;"></span><span class="mord"><span class="mord mathdefault" style="margin-right:0.07153em;">C</span></span></span><span style="top:-3.6023300000000003em;"><span class="pstrut" style="height:3em;"></span><span class="accent-body" style="left:-0.16666em;"><span class="mord">~</span></span></span></span></span></span></span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.2805559999999999em;"><span style="top:-2.5500000000000003em;margin-right:0.05em;"><span class="pstrut" style="height:2.7em;"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mathdefault mtight">t</span></span></span></span><span class="vlist-s">&#x200B;</span></span><span class="vlist-r"><span class="vlist" style="height:0.15em;"><span></span></span></span></span></span></span></span></span></span>&#x751F;&#x6210;&#x65B0;&#x7684;&#x5019;&#x9009;&#x503C;&#x3002;</p>
<p><img src="./images/rnn_lstm_focus-C.png" alt="&#x66F4;&#x65B0;&#x65E7;&#x5355;&#x5143;&#x72B6;&#x6001;"></p>
<h4 class="mume-header" id="%E8%BE%93%E5%87%BA%E9%97%A8">&#x8F93;&#x51FA;&#x95E8;</h4>

<p>&#x6700;&#x540E;&#xFF0C;&#x6211;&#x4EEC;&#x9700;&#x8981;&#x51B3;&#x5B9A;&#x8981;&#x8F93;&#x51FA;&#x7684;&#x5185;&#x5BB9;&#x3002;</p>
<p><img src="./images/rnn_lstm_focus-o.png" alt="&#x8F93;&#x51FA;&#x95E8;"></p>
<h3 class="mume-header" id="lstm%E5%8F%98%E4%BD%93">LSTM&#x53D8;&#x4F53;</h3>

<p>&#x7531;<a href="ftp://ftp.idsia.ch/pub/juergen/TimeCount-IJCNN2000.pdf">Gers &amp; Schmidhuber (2000)</a>&#x63D0;&#x51FA;&#xFF0C;&#x53EF;&#x4EE5;&#x8BA9;<strong>Gate Layers</strong>&#x67E5;&#x770B;&#x5355;&#x5143;&#x72B6;&#x6001;&#x3002;</p>
<p><img src="./images/rnn_lstm_var_peepholes.png" alt></p>
<p>&#x4F7F;&#x7528;&#x8026;&#x5408;&#x7684;<strong>&#x5FD8;&#x8BB0;&#x95E8;</strong>&#x548C;<strong>&#x8F93;&#x5165;&#x95E8;</strong>&#x3002;</p>
<p><img src="./images/rnn_lstm_var_tied.png" alt></p>
<p>&#x7531;<a href="http://arxiv.org/pdf/1406.1078v3.pdf">Cho, et al. (2014).</a>&#x63D0;&#x51FA;&#x7684;&#x95E8;&#x63A7;&#x5FAA;&#x73AF;&#x5355;&#x5143;&#xFF0C;&#x5C06;<strong>&#x5FD8;&#x8BB0;&#x95E8;</strong>&#x548C;<strong>&#x8F93;&#x5165;&#x95E8;</strong>&#x5408;&#x5E76;&#x4E3A;<strong>&#x66F4;&#x65B0;&#x95E8;</strong>&#xFF0C;&#x6B64;&#x5916;&#x8FD8;&#x5408;&#x5E76;&#x4E86;&#x5355;&#x5143;&#x72B6;&#x6001;&#x548C;&#x9690;&#x85CF;&#x72B6;&#x6001;&#x3002;</p>
<p><img src="./images/rnn_lstm_var_GRU.png" alt></p>

      </div>
      
      
    
    
    <script>
// config mermaid init call
// http://knsv.github.io/mermaid/#configuration
//
// You can edit the 'MERMAID_CONFIG' variable below.
MERMAID_CONFIG = {
  startOnLoad: false
}

if (window['MERMAID_CONFIG']) {
  window['MERMAID_CONFIG'].startOnLoad = false
  window['MERMAID_CONFIG'].cloneCssStyles = false
  window['MERMAID_CONFIG'].theme = "default"
}
mermaid.initialize(window['MERMAID_CONFIG'] || {})
if (typeof(window['Reveal']) !== 'undefined') {
  function mermaidRevealHelper(event) {
    var currentSlide = event.currentSlide
    var diagrams = currentSlide.querySelectorAll('.mermaid')
    for (var i = 0; i < diagrams.length; i++) {
      var diagram = diagrams[i]
      if (!diagram.hasAttribute('data-processed')) {
        mermaid.init(null, diagram, ()=> {
          Reveal.slide(event.indexh, event.indexv)
        })
      }
    }
  }
  Reveal.addEventListener('slidechanged', mermaidRevealHelper)
  Reveal.addEventListener('ready', mermaidRevealHelper)
} else {
  mermaid.init(null, document.getElementsByClassName('mermaid'))
}
</script>
    
    
    
    
    
  
    </body></html>